/*
    SUSAN® - Sole of Unix Save ANything

   Copyright (C) 2011-2016 Skyatlas Co. LTD
   

   
*/
/*
 * Marco van Wieringen, May 2015
 */
/**
 * @file
 * Backup specific NDMP Data Management Application (DMA) routines
 */

#include "include/susan.h"
#include "dird.h"
#include "dird/dird_globals.h"
#include "dird/jcr_private.h"
#include "dird/job.h"
#include "dird/msgchan.h"
#include "dird/quota.h"
#include "dird/sd_cmds.h"
#include "dird/storage.h"
#include "lib/edit.h"

#if HAVE_NDMP
#include "dird/ndmp_dma_backup_common.h"
#include "dird/ndmp_dma_generic.h"

#define NDMP_NEED_ENV_KEYWORDS 1

#include "ndmp/ndmagents.h"
#include "ndmp_dma_priv.h"
#endif /* HAVE_NDMP */

namespace directordaemon {

#if HAVE_NDMP

static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;

/* Imported variables */

/* Forward referenced functions */


/**
 * Extract any post backup statistics.
 */
static inline bool extract_post_backup_stats(JobControlRecord* jcr,
                                             char* filesystem,
                                             struct ndm_session* sess)
{
  bool retval = true;
  struct ndmmedia* media;
  ndmp_backup_format_option* nbf_options;
  struct ndm_env_entry* ndm_ee;

  /*
   * See if we know this backup format and get it options.
   */
  nbf_options =
      ndmp_lookup_backup_format_options(sess->control_acb->job.bu_type);

  /*
   * See if an error was raised during the backup session.
   */
  if (sess->error_raised) { return false; }

  /*
   * See if there is any media error.
   */
  for (media = sess->control_acb->job.result_media_tab.head; media;
       media = media->next) {
    if (media->media_open_error || media->media_io_error ||
        media->label_io_error || media->label_mismatch || media->fmark_error) {
      retval = false;
    }
  }

  /*
   * Process the FHDB.
   */
  ProcessFhdb(&sess->control_acb->job.index_log);

  /*
   * Update the Job statistics from the NDMP statistics.
   */
  jcr->JobBytes += sess->control_acb->job.bytes_written;

  /*
   * After a successful backup we need to store all NDMP ENV variables
   * for doing a successful restore operation.
   */
  ndm_ee = sess->control_acb->job.result_env_tab.head;
  while (ndm_ee) {
    if (!jcr->db->CreateNdmpEnvironmentString(
            jcr, &jcr->impl->jr, ndm_ee->pval.name, ndm_ee->pval.value)) {
      break;
    }
    ndm_ee = ndm_ee->next;
  }

  /*
   * If we are doing a backup type that uses dumplevels save the last used dump
   * level.
   */
  if (nbf_options && nbf_options->uses_level) {
    jcr->db->UpdateNdmpLevelMapping(jcr, &jcr->impl->jr, filesystem,
                                    sess->control_acb->job.bu_level);
  }

  return retval;
}

/**
 * Setup a NDMP backup session.
 */
bool DoNdmpBackupInit(JobControlRecord* jcr)
{
  FreeRstorage(jcr); /* we don't read so release */

  if (!AllowDuplicateJob(jcr)) { return false; }

  jcr->impl->jr.PoolId =
      GetOrCreatePoolRecord(jcr, jcr->impl->res.pool->resource_name_);
  if (jcr->impl->jr.PoolId == 0) { return false; }

  jcr->start_time = time(NULL);
  jcr->impl->jr.StartTime = jcr->start_time;
  if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
    Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
    return false;
  }

  /*
   * If pool storage specified, use it instead of job storage
   */
  CopyWstorage(jcr, jcr->impl->res.pool->storage, _("Pool resource"));

  if (!jcr->impl->res.write_storage_list) {
    Jmsg(jcr, M_FATAL, 0,
         _("No Storage specification found in Job or Pool.\n"));
    return false;
  }

  /*
   * Validate the Job to have a NDMP client and NDMP storage.
   */
  if (!NdmpValidateClient(jcr)) { return false; }

  if (!NdmpValidateStorage(jcr)) { return false; }

  /*
   * For now we only allow NDMP backups to susan SD's
   * so we need a paired storage definition.
   */
  if (!HasPairedStorage(jcr)) {
    Jmsg(jcr, M_FATAL, 0,
         _("Write storage doesn't point to storage definition with paired "
           "storage option.\n"));
    return false;
  }

  return true;
}


/**
 * Run a NDMP backup session.
 */
bool DoNdmpBackup(JobControlRecord* jcr)
{
  unsigned int cnt;
  int status;
  unsigned int i;
  char ed1[100];
  NIS* nis = NULL;
  FilesetResource* fileset;
  struct ndm_job_param ndmp_job;
  struct ndm_session ndmp_sess;
  bool session_initialized = false;
  bool retval = false;
  int NdmpLoglevel;

  NdmpLoglevel =
      std::max(jcr->impl->res.client->ndmp_loglevel, me->ndmp_loglevel);

  /*
   * Print Job Start message
   */
  Jmsg(jcr, M_INFO, 0, _("Start NDMP Backup JobId %s, Job=%s\n"),
       edit_uint64(jcr->JobId, ed1), jcr->Job);

  jcr->setJobStatus(JS_Running);
  Dmsg2(100, "JobId=%d JobLevel=%c\n", jcr->impl->jr.JobId,
        jcr->impl->jr.JobLevel);
  if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
    Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
    return false;
  }

  if (CheckHardquotas(jcr)) {
    Jmsg(jcr, M_FATAL, 0, "Quota Exceeded. Job terminated.");
    return false;
  }

  if (CheckSoftquotas(jcr)) {
    Dmsg0(10, "Quota exceeded\n");
    Jmsg(jcr, M_FATAL, 0,
         "Soft Quota Exceeded / Grace Time expired. Job terminated.");
    return false;
  }

  /*
   * If we have a paired storage definition create a native connection
   * to a Storage daemon and make it ready to receive a backup.
   * The setup is more or less the same as for a normal non NDMP backup
   * only the data doesn't come in from a FileDaemon but from a NDMP
   * data mover which moves the data from the NDMP DATA AGENT to the NDMP
   * TAPE AGENT.
   */
  if (jcr->impl->res.write_storage->paired_storage) {
    SetPairedStorage(jcr);

    jcr->setJobStatus(JS_WaitSD);
    if (!ConnectToStorageDaemon(jcr, 10, me->SDConnectTimeout, true)) {
      return false;
    }

    /*
     * Now start a job with the Storage daemon
     */
    if (!StartStorageDaemonJob(jcr, NULL, jcr->impl->res.write_storage_list)) {
      return false;
    }

    /*
     * Start the job prior to starting the message thread below
     * to avoid two threads from using the SusanSocket structure at
     * the same time.
     */
    if (!jcr->store_bsock->fsend("run")) { return false; }

    /*
     * Now start a Storage daemon message thread.  Note,
     *   this thread is used to provide the catalog services
     *   for the backup job.
     */
    if (!StartStorageDaemonMessageThread(jcr)) { return false; }
    Dmsg0(150, "Storage daemon connection OK\n");
  }

  status = 0;

  /*
   * Initialize the ndmp backup job. We build the generic job only once
   * and reuse the job definition for each separate sub-backup we perform as
   * part of the whole job. We only free the env_table between every sub-backup.
   */
  if (!NdmpBuildClientJob(jcr, jcr->impl->res.client,
                          jcr->impl->res.paired_read_write_storage,
                          NDM_JOB_OP_BACKUP, &ndmp_job)) {
    goto bail_out;
  }

  nis = (NIS*)malloc(sizeof(NIS));
  memset(nis, 0, sizeof(NIS));

  /*
   * Loop over each include set of the fileset and fire off a NDMP backup of the
   * included fileset.
   */
  cnt = 0;
  fileset = jcr->impl->res.fileset;


  for (i = 0; i < fileset->include_items.size(); i++) {
    int j;
    char* item;
    IncludeExcludeItem* ie = fileset->include_items[i];
    PoolMem virtual_filename(PM_FNAME);

    /*
     * Loop over each file = entry of the fileset.
     */
    for (j = 0; j < ie->name_list.size(); j++) {
      item = (char*)ie->name_list.get(j);

      /*
       * See if this is the first Backup run or not. For NDMP we can have
       * multiple Backup runs as part of the same Job. When we are saving data
       * to a Native Storage Daemon we let it know to expect a new backup
       * session. It will generate a new authorization key so we wait for the
       * nextrun_ready conditional variable to be raised by the msg_thread.
       */
      if (jcr->store_bsock && cnt > 0) {
        jcr->store_bsock->fsend("nextrun");
        P(mutex);
        pthread_cond_wait(&jcr->impl->nextrun_ready, &mutex);
        V(mutex);
      }

      /*
       * Perform the actual NDMP job.
       * Initialize a new NDMP session
       */
      memset(&ndmp_sess, 0, sizeof(ndmp_sess));
      ndmp_sess.conn_snooping = (me->ndmp_snooping) ? 1 : 0;
      ndmp_sess.control_agent_enabled = 1;

      ndmp_sess.param =
          (struct ndm_session_param*)malloc(sizeof(struct ndm_session_param));
      memset(ndmp_sess.param, 0, sizeof(struct ndm_session_param));
      ndmp_sess.param->log.deliver = NdmpLoghandler;
      ndmp_sess.param->log_level =
          NativeToNdmpLoglevel(NdmpLoglevel, debug_level, nis);
      nis->filesystem = item;
      nis->FileIndex = cnt + 1;
      nis->jcr = jcr;
      nis->save_filehist = jcr->impl->res.job->SaveFileHist;
      nis->filehist_size = jcr->impl->res.job->FileHistSize;

      ndmp_sess.param->log.ctx = nis;
      ndmp_sess.param->log_tag = strdup("DIR-NDMP");

      /*
       * Initialize the session structure.
       */
      if (ndma_session_initialize(&ndmp_sess)) { goto cleanup; }
      session_initialized = true;

      /*
       * Copy the actual job to perform.
       */
      memcpy(&ndmp_sess.control_acb->job, &ndmp_job,
             sizeof(struct ndm_job_param));

      /*
       * We can use the same private pointer used in the logging with the
       * JobControlRecord in the file index generation. We don't setup a
       * index_log.deliver function as we catch the index information via
       * callbacks.
       */
      ndmp_sess.control_acb->job.index_log.ctx = ndmp_sess.param->log.ctx;

      if (!FillBackupEnvironment(jcr, ie, nis->filesystem,
                                 &ndmp_sess.control_acb->job)) {
        goto cleanup;
      }

      /*
       * The full ndmp archive has a virtual filename, we need it to hardlink
       * the individual file records to it. So we allocate it here once so its
       * available during the whole NDMP session.
       */
      if (Bstrcasecmp(jcr->impl->backup_format, "dump")) {
        Mmsg(virtual_filename, "/@NDMP%s%%%d", nis->filesystem,
             jcr->impl->DumpLevel);
      } else {
        Mmsg(virtual_filename, "/@NDMP%s", nis->filesystem);
      }

      if (nis->virtual_filename) { free(nis->virtual_filename); }
      nis->virtual_filename = strdup(virtual_filename.c_str());

      ndma_job_auto_adjust(&ndmp_sess.control_acb->job);
      if (!NdmpValidateJob(jcr, &ndmp_sess.control_acb->job)) { goto cleanup; }

      /*
       * Commission the session for a run.
       */
      if (ndma_session_commission(&ndmp_sess)) { goto cleanup; }

      /*
       * Setup the DMA.
       */
      if (ndmca_connect_control_agent(&ndmp_sess)) { goto cleanup; }

      ndmp_sess.conn_open = 1;
      ndmp_sess.conn_authorized = 1;

      RegisterCallbackHooks(&ndmp_sess.control_acb->job.index_log);

      /*
       * Let the DMA perform its magic.
       */
      if (ndmca_control_agent(&ndmp_sess) != 0) { goto cleanup; }

      /*
       * See if there were any errors during the backup.
       */
      jcr->impl->jr.FileIndex = cnt + 1;
      if (!extract_post_backup_stats(jcr, item, &ndmp_sess)) { goto cleanup; }

      UnregisterCallbackHooks(&ndmp_sess.control_acb->job.index_log);

      /*
       * Reset the NDMP session states.
       */
      ndma_session_decommission(&ndmp_sess);

      /*
       * Cleanup the job after it has run.
       */
      ndma_destroy_env_list(&ndmp_sess.control_acb->job.env_tab);
      ndma_destroy_env_list(&ndmp_sess.control_acb->job.result_env_tab);
      ndma_destroy_nlist(&ndmp_sess.control_acb->job.nlist_tab);

      /*
       * Release any tape device name allocated.
       */
      if (ndmp_sess.control_acb->job.tape_device) {
        free(ndmp_sess.control_acb->job.tape_device);
        ndmp_sess.control_acb->job.tape_device = NULL;
      }

      /*
       * Destroy the session.
       */
      ndma_session_destroy(&ndmp_sess);

      /*
       * Free the param block.
       */
      free(ndmp_sess.param->log_tag);
      free(ndmp_sess.param);
      ndmp_sess.param = NULL;

      /*
       * Reset the initialized state so we don't try to cleanup again.
       */
      session_initialized = false;

      cnt++;
    }
  }

  status = JS_Terminated;
  retval = true;

  /*
   * Tell the storage daemon we are done.
   */
  if (jcr->store_bsock) {
    jcr->store_bsock->fsend("finish");
    WaitForStorageDaemonTermination(jcr);
    jcr->db_batch->WriteBatchFileRecords(
        jcr); /* used by bulk batch file insert */
  }

  /*
   * If we do incremental backups it can happen that the backup is empty if
   * nothing changed but we always write a filestream. So we use the counter
   * which counts the number of actual NDMP backup sessions we run to
   * completion.
   */
  if (jcr->JobFiles < cnt) { jcr->JobFiles = cnt; }

  /*
   * Jump to the generic cleanup done for every Job.
   */
  goto ok_out;

cleanup:
  /*
   * Only need to cleanup when things are initialized.
   */
  if (session_initialized) {
    ndma_destroy_env_list(&ndmp_sess.control_acb->job.env_tab);
    ndma_destroy_env_list(&ndmp_sess.control_acb->job.result_env_tab);
    ndma_destroy_nlist(&ndmp_sess.control_acb->job.nlist_tab);

    if (ndmp_sess.control_acb->job.tape_device) {
      free(ndmp_sess.control_acb->job.tape_device);
    }

    UnregisterCallbackHooks(&ndmp_sess.control_acb->job.index_log);

    /*
     * Destroy the session.
     */
    ndma_session_destroy(&ndmp_sess);
  }

  if (ndmp_sess.param) {
    free(ndmp_sess.param->log_tag);
    free(ndmp_sess.param);
  }

bail_out:
  /*
   * Error handling of failed Job.
   */
  status = JS_ErrorTerminated;
  jcr->setJobStatus(JS_ErrorTerminated);

  if (jcr->store_bsock) {
    CancelStorageDaemonJob(jcr);
    WaitForStorageDaemonTermination(jcr);
  }

ok_out:
  if (nis) {
    if (nis->virtual_filename) { free(nis->virtual_filename); }
    free(nis);
  }
  FreePairedStorage(jcr);

  if (status == JS_Terminated) { NdmpBackupCleanup(jcr, status); }

  return retval;
}

#else
bool DoNdmpBackupInit(JobControlRecord* jcr)
{
  Jmsg(jcr, M_FATAL, 0, _("NDMP protocol not supported\n"));
  return false;
}

bool DoNdmpBackup(JobControlRecord* jcr)
{
  Jmsg(jcr, M_FATAL, 0, _("NDMP protocol not supported\n"));
  return false;
}

#endif /* HAVE_NDMP */
} /* namespace directordaemon */
